{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:16.487931Z",
     "start_time": "2017-10-26T00:32:15.306107Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "import re\n",
    "import collections\n",
    "import itertools\n",
    "import bcolz\n",
    "import pickle\n",
    "sys.path.append('../../lib')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import random\n",
    "import smart_open\n",
    "import h5py\n",
    "import csv\n",
    "import json\n",
    "import functools\n",
    "import time\n",
    "import string\n",
    "\n",
    "import datetime as dt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import global_utils\n",
    "\n",
    "random_state_number = 967898"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:16.985230Z",
     "start_time": "2017-10-26T00:32:16.489185Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/gpu:0', '/gpu:1']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)\n",
    "get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:17.074281Z",
     "start_time": "2017-10-26T00:32:16.986890Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: TkAgg\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bicepjai/Programs/anaconda3/envs/dsotc-c3/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['random']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    }
   ],
   "source": [
    "%pylab\n",
    "%matplotlib inline\n",
    "%load_ext line_profiler\n",
    "%load_ext memory_profiler\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:17.078729Z",
     "start_time": "2017-10-26T00:32:17.075713Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment = None\n",
    "pd.options.display.max_columns = 999\n",
    "color = sns.color_palette()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:22:31.466526Z",
     "start_time": "2017-10-26T21:22:31.455325Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "store = pd.HDFStore('../../data_prep/processed/stage1/data_frames.h5')\n",
    "train_df = store['train_df']\n",
    "test_df = store['test_df']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:35.739219Z",
     "start_time": "2017-10-26T00:32:35.425475Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Class</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[fam58a]</td>\n",
       "      <td>[truncating, mutations]</td>\n",
       "      <td>1</td>\n",
       "      <td>[[cyclin-dependent, kinases, , cdks, , regulat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[w802*]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[q249e]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[n454d]</td>\n",
       "      <td>3</td>\n",
       "      <td>[[recent, evidence, has, demonstrated, that, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[l399v]</td>\n",
       "      <td>4</td>\n",
       "      <td>[[oncogenic, mutations, in, the, monomeric, ca...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID      Gene                Variation  Class  \\\n",
       "0   0  [fam58a]  [truncating, mutations]      1   \n",
       "1   1     [cbl]                  [w802*]      2   \n",
       "2   2     [cbl]                  [q249e]      2   \n",
       "3   3     [cbl]                  [n454d]      3   \n",
       "4   4     [cbl]                  [l399v]      4   \n",
       "\n",
       "                                           Sentences  \n",
       "0  [[cyclin-dependent, kinases, , cdks, , regulat...  \n",
       "1  [[abstract, background, non-small, cell, lung,...  \n",
       "2  [[abstract, background, non-small, cell, lung,...  \n",
       "3  [[recent, evidence, has, demonstrated, that, a...  \n",
       "4  [[oncogenic, mutations, in, the, monomeric, ca...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[acsl4]</td>\n",
       "      <td>[r570s]</td>\n",
       "      <td>[[2, this, mutation, resulted, in, a, myelopro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[naglu]</td>\n",
       "      <td>[p521l]</td>\n",
       "      <td>[[abstract, the, large, tumor, suppressor, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[pah]</td>\n",
       "      <td>[l333f]</td>\n",
       "      <td>[[vascular, endothelial, growth, factor, recep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[ing1]</td>\n",
       "      <td>[a148d]</td>\n",
       "      <td>[[inflammatory, myofibroblastic, tumor, , imt,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[tmem216]</td>\n",
       "      <td>[g77a]</td>\n",
       "      <td>[[abstract, retinoblastoma, is, a, pediatric, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID       Gene Variation                                          Sentences\n",
       "0   0    [acsl4]   [r570s]  [[2, this, mutation, resulted, in, a, myelopro...\n",
       "1   1    [naglu]   [p521l]  [[abstract, the, large, tumor, suppressor, 1, ...\n",
       "2   2      [pah]   [l333f]  [[vascular, endothelial, growth, factor, recep...\n",
       "3   3     [ing1]   [a148d]  [[inflammatory, myofibroblastic, tumor, , imt,...\n",
       "4   4  [tmem216]    [g77a]  [[abstract, retinoblastoma, is, a, pediatric, ..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(train_df.head())\n",
    "display(test_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:35.843893Z",
     "start_time": "2017-10-26T00:32:35.740546Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "352220 352220\n"
     ]
    }
   ],
   "source": [
    "corpus_vocab_list, corpus_vocab_wordidx = None, None\n",
    "with open('../../data_prep/processed/stage1/vocab_words_wordidx.pkl', 'rb') as f:\n",
    "    (corpus_vocab_list, corpus_wordidx) = pickle.load(f)\n",
    "print(len(corpus_vocab_list), len(corpus_wordidx))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-14T08:20:17.449244Z",
     "start_time": "2017-08-14T08:20:15.593136Z"
    },
    "collapsed": true
   },
   "source": [
    "# Data Prep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To control the vocabulary pass in updated corpus_wordidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:35.881731Z",
     "start_time": "2017-10-26T00:32:35.845026Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5)\n",
      "(333, 5)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train_df, x_val_df = train_test_split(train_df,\n",
    "                                         test_size=0.10, random_state=random_state_number,\n",
    "                                         stratify=train_df.Class)\n",
    "\n",
    "print(x_train_df.shape)\n",
    "print(x_val_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:43.169601Z",
     "start_time": "2017-10-26T00:32:35.883008Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.contrib.keras.python.keras.utils import np_utils\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:43.172882Z",
     "start_time": "2017-10-26T00:32:43.170730Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_size=len(corpus_vocab_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## T:sent_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:14:20.659301Z",
     "start_time": "2017-10-26T00:14:20.653375Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"sentences\",\n",
    "         \"divide_document\": \"multiple_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:16:51.148007Z",
     "start_time": "2017-10-26T00:14:22.265259Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_21_T, x_train_21_G, x_train_21_V, x_train_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:16:55.113872Z",
     "start_time": "2017-10-26T00:16:51.149180Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "(2622081,) [364606, 113692, 197002, 330024, 326252, 151042, 75648, 1818, 276247, 61043, 228115, 326252, 74974, 301275, 76659, 326252, 361104, 329709, 253643, 205596, 153283, 326252, 80594, 326252, 113692, 18820, 349251, 59442, 123801, 228752, 245229, 307200, 17105, 60555, 69032, 1818, 274163, 151942, 246684, 222367, 253643, 243777, 274163, 50915, 274163, 12413, 1818, 228752, 364603, 232434, 214275, 235155, 163151, 123801, 101614, 101366, 364607]\n",
      "(2622081, 3) [364606, 97957, 364607]\n",
      "(2622081,) [364606, 326252, 364607]\n",
      "(2622081,) 6\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(np.array(x_train_21_T).shape, x_train_21_T[0])\n",
    "print(np.array(x_train_21_G).shape, x_train_21_G[0])\n",
    "print(np.array(x_train_21_V).shape, x_train_21_V[0])\n",
    "print(np.array(x_train_21_C).shape, x_train_21_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.385302Z",
     "start_time": "2017-10-26T00:16:55.114982Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_21_T, x_val_21_G, x_val_21_V, x_val_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.853790Z",
     "start_time": "2017-10-26T00:17:10.386619Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (293702,)\n",
      "gene (293702, 3) [364606, 112978, 364607]\n",
      "variation (293702,) [364606, 295010, 364607]\n",
      "classes (293702,) 2\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_21_T).shape)\n",
    "print(\"gene\",np.array(x_val_21_G).shape, x_val_21_G[0])\n",
    "print(\"variation\",np.array(x_val_21_V).shape, x_val_21_V[0])\n",
    "print(\"classes\",np.array(x_val_21_C).shape, x_val_21_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.857143Z",
     "start_time": "2017-10-26T00:17:10.854958Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.872469Z",
     "start_time": "2017-10-26T00:17:10.858162Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "MAX_SENT_LEN = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.327704Z",
     "start_time": "2017-10-26T00:17:10.873609Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2622081, 60) (293702, 60)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_T = pad_sequences(x_train_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_21_T = pad_sequences(x_val_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_21_T.shape, x_val_21_T.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.564842Z",
     "start_time": "2017-10-26T00:17:27.328915Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_train_21_C = np.array(x_train_21_C) - 1\n",
    "x_val_21_C = np.array(x_val_21_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.631996Z",
     "start_time": "2017-10-26T00:17:27.566066Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2622081, 9) (293702, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_C = np_utils.to_categorical(np.array(x_train_21_C), 9)\n",
    "x_val_21_C = np_utils.to_categorical(np.array(x_val_21_C), 9)\n",
    "print(x_train_21_C.shape, x_val_21_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## T:text_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:43.191953Z",
     "start_time": "2017-10-26T00:32:43.173899Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"text\",\n",
    "         \"divide_document\": \"single_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:53.910517Z",
     "start_time": "2017-10-26T00:32:43.193444Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_22_T, x_train_22_G, x_train_22_V, x_train_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:54.978928Z",
     "start_time": "2017-10-26T00:32:53.911691Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "text (2988,)\n",
      "gene (2988, 3) [352216, 164788, 352217]\n",
      "variation (2988,) [352216, 86196, 352217]\n",
      "classes (2988,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(\"text\",np.array(x_train_22_T).shape)\n",
    "print(\"gene\",np.array(x_train_22_G).shape, x_train_22_G[0])\n",
    "print(\"variation\",np.array(x_train_22_V).shape, x_train_22_V[0])\n",
    "print(\"classes\",np.array(x_train_22_C).shape, x_train_22_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:56.258929Z",
     "start_time": "2017-10-26T00:32:54.980078Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_22_T, x_val_22_G, x_val_22_V, x_val_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:56.411716Z",
     "start_time": "2017-10-26T00:32:56.260210Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (333,)\n",
      "gene (333, 3) [352216, 217983, 352217]\n",
      "variation (333,) [352216, 41934, 352217]\n",
      "classes (333,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_22_T).shape)\n",
    "print(\"gene\",np.array(x_val_22_G).shape, x_val_22_G[0])\n",
    "print(\"variation\",np.array(x_val_22_V).shape, x_val_22_V[0])\n",
    "print(\"classes\",np.array(x_val_22_C).shape, x_val_22_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:56.437499Z",
     "start_time": "2017-10-26T00:32:56.413156Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:56.450804Z",
     "start_time": "2017-10-26T00:32:56.439121Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_TEXT_LEN = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.501666Z",
     "start_time": "2017-10-26T00:32:56.452013Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5000) (333, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_T = pad_sequences(x_train_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_22_T = pad_sequences(x_val_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_22_T.shape, x_val_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.536590Z",
     "start_time": "2017-10-26T00:32:57.502761Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 1) (2988, 4)\n",
      "(333, 1) (333, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_train_22_G = pad_sequences(x_train_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_train_22_V = pad_sequences(x_train_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "x_val_22_G = pad_sequences(x_val_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_val_22_V = pad_sequences(x_val_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_train_22_G.shape, x_train_22_V.shape)\n",
    "print(x_val_22_G.shape, x_val_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.542037Z",
     "start_time": "2017-10-26T00:32:57.538132Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train_22_C = np.array(x_train_22_C) - 1\n",
    "x_val_22_C = np.array(x_val_22_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.576726Z",
     "start_time": "2017-10-26T00:32:57.543764Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 9) (333, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_C = np_utils.to_categorical(np.array(x_train_22_C), 9)\n",
    "x_val_22_C = np_utils.to_categorical(np.array(x_val_22_C), 9)\n",
    "print(x_train_22_C.shape, x_val_22_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### test Data setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:32.887420Z",
     "start_time": "2017-09-26T03:43:29.372697Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(test_df, corpus_wordidx)\n",
    "x_test_22_T, x_test_22_G, x_test_22_V, _ = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                has_class=False,\n",
    "                                                                add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.178763Z",
     "start_time": "2017-09-26T03:43:32.888877Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test data\n",
      "text (986,)\n",
      "gene (986, 3) [364606, 188717, 364607]\n",
      "variation (986,) [364606, 317947, 364607]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test data\")\n",
    "print(\"text\",np.array(x_test_22_T).shape)\n",
    "print(\"gene\",np.array(x_test_22_G).shape, x_test_22_G[0])\n",
    "print(\"variation\",np.array(x_test_22_V).shape, x_test_22_V[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.546461Z",
     "start_time": "2017-09-26T03:43:33.181689Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(986, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_test_22_T = pad_sequences(x_test_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_test_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.575305Z",
     "start_time": "2017-09-26T03:43:33.548386Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(986, 1) (986, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_test_22_G = pad_sequences(x_test_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_test_22_V = pad_sequences(x_test_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_test_22_G.shape, x_test_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:32:57.595494Z",
     "start_time": "2017-10-26T00:32:57.579320Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "WORD_EMB_SIZE = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:33:20.867764Z",
     "start_time": "2017-10-26T00:32:57.597200Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(352220, 200)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "ft_file_path = \"/home/bicepjai/Projects/Deep-Survey-Text-Classification/data_prep/processed/stage1/pretrained_word_vectors/ft_sg_200d_50e.vec\"\n",
    "trained_embeddings = global_utils.get_embeddings_from_ft(ft_file_path, WORD_EMB_SIZE, corpus_vocab_list)\n",
    "trained_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### for characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:04:28.473180Z",
     "start_time": "2017-09-25T06:04:28.470711Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "CHAR_EMB_SIZE = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:04:28.479331Z",
     "start_time": "2017-09-25T06:04:28.474704Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 100)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_embeddings = np.random.randn(global_utils.CHAR_ALPHABETS_LEN, CHAR_EMB_SIZE)\n",
    "char_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:33:20.908977Z",
     "start_time": "2017-10-26T00:33:20.869343Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow.contrib.keras as keras\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.engine import Layer, InputSpec, InputLayer\n",
    "\n",
    "from keras.models import Model, Sequential\n",
    "\n",
    "from keras.layers import Dropout, Embedding, concatenate\n",
    "from keras.layers import Conv1D, MaxPool1D, Conv2D, MaxPool2D, ZeroPadding1D\n",
    "from keras.layers import Dense, Input, Flatten, BatchNormalization\n",
    "from keras.layers import Concatenate, Dot, Merge, Multiply, RepeatVector\n",
    "from keras.layers import Bidirectional, TimeDistributed\n",
    "from keras.layers import SimpleRNN, LSTM, GRU, Lambda, Permute\n",
    "\n",
    "from keras.layers.core import Reshape, Activation\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint,EarlyStopping,TensorBoard\n",
    "from keras.constraints import maxnorm\n",
    "from keras.regularizers import l2\n",
    "\n",
    "%autoreload"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KMaxPooling layer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:33:20.918486Z",
     "start_time": "2017-10-26T00:33:20.910344Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from utils import KMaxPooling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Folding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:33:20.938089Z",
     "start_time": "2017-10-26T00:33:20.920166Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from utils import Folding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T06:58:17.661183Z",
     "start_time": "2017-08-24T06:58:17.655020Z"
    }
   },
   "source": [
    "## model_1: paper"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " CNN with Dynamic k-Max Pooling with sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:23:23.190159Z",
     "start_time": "2017-10-26T00:23:21.997528Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_1 = Sequential([\n",
    "    Embedding(vocab_size, WORD_EMB_SIZE, weights=[trained_embeddings],\n",
    "              input_length=MAX_TEXT_LEN,trainable=True),\n",
    "    ZeroPadding1D((49,49)),\n",
    "    Conv1D(64, 50, padding=\"same\"),\n",
    "    KMaxPooling(k=5, axis=1),\n",
    "    Activation(\"relu\"),\n",
    "    ZeroPadding1D((24,24)),\n",
    "    Conv1D(64, 25, padding=\"same\"),\n",
    "    Folding(),\n",
    "    KMaxPooling(k=5, axis=1),\n",
    "    Activation(\"relu\"),\n",
    "    Flatten(),\n",
    "    Dense(9, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:23:23.219070Z",
     "start_time": "2017-10-26T00:23:23.191474Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 5000, 200)         70444000  \n",
      "_________________________________________________________________\n",
      "zero_padding1d_1 (ZeroPaddin (None, 5098, 200)         0         \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 5098, 64)          640064    \n",
      "_________________________________________________________________\n",
      "k_max_pooling_1 (KMaxPooling (None, 5, 64)             0         \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 5, 64)             0         \n",
      "_________________________________________________________________\n",
      "zero_padding1d_2 (ZeroPaddin (None, 53, 64)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 53, 64)            102464    \n",
      "_________________________________________________________________\n",
      "folding_1 (Folding)          (None, 53, 32)            0         \n",
      "_________________________________________________________________\n",
      "k_max_pooling_2 (KMaxPooling (None, 5, 32)             0         \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 5, 32)             0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 160)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 9)                 1449      \n",
      "=================================================================\n",
      "Total params: 71,187,977\n",
      "Trainable params: 71,187,977\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_1.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])\n",
    "model_1.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:24:20.079839Z",
     "start_time": "2017-10-26T00:24:20.074483Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:25:18.165488Z",
     "start_time": "2017-10-26T00:25:18.159937Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(filepath=\"model_1_weights.hdf5\", \n",
    "                                    verbose=1,\n",
    "                                    monitor=\"val_categorical_accuracy\",\n",
    "                                    save_best_only=True,\n",
    "                                    mode=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:25:18.556945Z",
     "start_time": "2017-10-26T00:25:18.553156Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "earlystopping = EarlyStopping(monitor='val_categorical_accuracy', \n",
    "                              min_delta=0, patience=5, \n",
    "                              verbose=1, mode='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:27:04.039750Z",
     "start_time": "2017-10-26T00:25:19.225244Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no checkpoints available !\n",
      "Train on 2988 samples, validate on 333 samples\n",
      "Epoch 1/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 1.8754 - categorical_accuracy: 0.2972Epoch 00000: val_categorical_accuracy improved from -inf to 0.37538, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 12s - loss: 1.8752 - categorical_accuracy: 0.2975 - val_loss: 1.7255 - val_categorical_accuracy: 0.3754\n",
      "Epoch 2/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 1.5359 - categorical_accuracy: 0.4708Epoch 00001: val_categorical_accuracy improved from 0.37538 to 0.48949, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 10s - loss: 1.5339 - categorical_accuracy: 0.4726 - val_loss: 1.4433 - val_categorical_accuracy: 0.4895\n",
      "Epoch 3/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 1.1139 - categorical_accuracy: 0.6539Epoch 00002: val_categorical_accuracy improved from 0.48949 to 0.62162, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 10s - loss: 1.1129 - categorical_accuracy: 0.6543 - val_loss: 1.2026 - val_categorical_accuracy: 0.6216\n",
      "Epoch 4/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.8395 - categorical_accuracy: 0.7323Epoch 00003: val_categorical_accuracy improved from 0.62162 to 0.63664, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 10s - loss: 0.8383 - categorical_accuracy: 0.7329 - val_loss: 1.1661 - val_categorical_accuracy: 0.6366\n",
      "Epoch 5/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.7113 - categorical_accuracy: 0.7660Epoch 00004: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 8s - loss: 0.7139 - categorical_accuracy: 0.7644 - val_loss: 1.1597 - val_categorical_accuracy: 0.6186\n",
      "Epoch 6/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.6362 - categorical_accuracy: 0.7758Epoch 00005: val_categorical_accuracy improved from 0.63664 to 0.64264, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 10s - loss: 0.6350 - categorical_accuracy: 0.7751 - val_loss: 1.2112 - val_categorical_accuracy: 0.6426\n",
      "Epoch 7/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.5894 - categorical_accuracy: 0.7836Epoch 00006: val_categorical_accuracy improved from 0.64264 to 0.66366, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 10s - loss: 0.5873 - categorical_accuracy: 0.7845 - val_loss: 1.1550 - val_categorical_accuracy: 0.6637\n",
      "Epoch 8/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.5814 - categorical_accuracy: 0.7802Epoch 00007: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 8s - loss: 0.5830 - categorical_accuracy: 0.7805 - val_loss: 1.2309 - val_categorical_accuracy: 0.6186\n",
      "Epoch 9/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.5473 - categorical_accuracy: 0.7891Epoch 00008: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 8s - loss: 0.5453 - categorical_accuracy: 0.7895 - val_loss: 1.1817 - val_categorical_accuracy: 0.6486\n",
      "Epoch 10/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.5308 - categorical_accuracy: 0.7819Epoch 00009: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 9s - loss: 0.5303 - categorical_accuracy: 0.7818 - val_loss: 1.1884 - val_categorical_accuracy: 0.6186\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_1.load_weights(\"model_1_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    model_1.fit(x_train_22_T, x_train_22_C, \n",
    "          validation_data=(x_val_22_T, x_val_22_C),\n",
    "          epochs=10, batch_size=128,shuffle=True,\n",
    "          callbacks=[tb_callback,checkpointer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "913px",
    "left": "0px",
    "right": "1192px",
    "top": "52px",
    "width": "300px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
